{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "from bs4 import BeautifulSoup\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import lightgbm as lgb\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "stemmer = WordNetLemmatizer()\n",
    "from scipy import sparse\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.ticker import StrMethodFormatter\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from math import sqrt\n",
    "from sklearn.decomposition import TruncatedSVD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('C:\\\\Kaggle\\\\BooksPrice\\\\Participants_Data\\\\Data_Train08.csv')\n",
    "test = pd.read_csv('C:\\\\Kaggle\\\\BooksPrice\\\\Participants_Data\\\\Data_Test08.csv')\n",
    "\n",
    "train['Price_Log'] = np.log10(train['Price']+1)\n",
    "train.hist(column='Price_Log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "FeatureNames = list(train.columns[4:])\n",
    "FeatureNames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fold_list = list(train.FOLD_NUM.unique())\n",
    "fold_list.sort()\n",
    "fold_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_cv_predictions = train[['id','FOLD_NUM','Price','Price_Log']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "param = {\n",
    "    'learning_rate': 0.001,\n",
    "    'max_depth': 10,\n",
    "    'min_data_in_leaf': 3,\n",
    "    'bagging_freq': 5,\n",
    "    'bagging_fraction': 0.3,\n",
    "    'feature_fraction': 0.3,\n",
    "    'boost': 'gbdt',    \n",
    "    'objective': 'regression',\n",
    "    'metric': 'rmse',\n",
    "    'seed': 534\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "IterationNum = 1\n",
    "for fold_num in fold_list:\n",
    "    print(\"Running CV Iteration Num :\", IterationNum)\n",
    "    \n",
    "    temp_train = train[train['FOLD_NUM'] != fold_num]\n",
    "    temp_val = train[train['FOLD_NUM'] == fold_num]\n",
    "    \n",
    "    temp_train_tf = temp_train[FeatureNames].values\n",
    "    temp_val_tf = temp_val[FeatureNames].values\n",
    "    \n",
    "    trn_data = lgb.Dataset(temp_train_tf, label=temp_train['Price_Log'])\n",
    "    val_data = lgb.Dataset(temp_val_tf, label=temp_val['Price_Log'])\n",
    "    \n",
    "    model = lgb.train(param, trn_data, 2000000,valid_sets = val_data, verbose_eval=50, early_stopping_rounds = 300)\n",
    "    \n",
    "    temp_val['Price_Log_Pred_LGB'] = model.predict(temp_val_tf, num_iteration=model.best_iteration)\n",
    "    \n",
    "    print(\"Fold RMSLE = \",sqrt(mean_squared_error(temp_val['Price_Log'], temp_val['Price_Log_Pred_LGB'])))\n",
    "    \n",
    "    if(IterationNum == 1):\n",
    "        CV_SCORED_DATA = temp_val.copy(deep=True)\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        sub_data_preds = model.predict(test[FeatureNames].values, num_iteration=model.best_iteration)\n",
    "    else:\n",
    "        CV_SCORED_DATA = pd.concat([CV_SCORED_DATA,temp_val])\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        sub_data_preds = sub_data_preds + model.predict(test[FeatureNames].values, num_iteration=model.best_iteration)\n",
    "\n",
    "    IterationNum = IterationNum + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"CV RMSLE = \",sqrt(mean_squared_error(CV_SCORED_DATA['Price_Log'], CV_SCORED_DATA['Price_Log_Pred_LGB'])))\n",
    "print(\"CV 1-RMSLE = \",1-sqrt(mean_squared_error(CV_SCORED_DATA['Price_Log'], CV_SCORED_DATA['Price_Log_Pred_LGB'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_cv_predictions = pd.merge(training_cv_predictions, CV_SCORED_DATA[['id','Price_Log_Pred_LGB']], on='id')\n",
    "training_cv_predictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_data_preds = sub_data_preds / len(fold_list)\n",
    "test['Price_Log_Pred_LGB'] = sub_data_preds\n",
    "test['Price_Log_Pred_LGB'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential, load_model\n",
    "from keras.layers import Dense, Dropout\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nn_model(input_shape):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(30, input_dim=input_shape, activation='elu'))\n",
    "    model.add(Dropout(0.05))\n",
    "    model.add(Dense(15, activation='elu'))\n",
    "    model.add(Dropout(0.05))\n",
    "    model.add(Dense(1))\n",
    "    return(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras01_Models = []\n",
    "IterationNum = 1\n",
    "batch_size = 64\n",
    "for fold_num in fold_list:\n",
    "    print(\"Running CV Iteration Num :\", IterationNum)\n",
    "    \n",
    "    temp_train = train[train['FOLD_NUM'] != fold_num]\n",
    "    temp_val = train[train['FOLD_NUM'] == fold_num]\n",
    "    \n",
    "    temp_train_tf = temp_train[FeatureNames].values\n",
    "    temp_val_tf = temp_val[FeatureNames].values\n",
    "    \n",
    "    model = nn_model(temp_train_tf.shape[1])\n",
    "    model.compile(loss = 'mean_squared_error', optimizer='adam',metrics = ['mean_squared_error'])\n",
    "    print(model.summary())\n",
    "    \n",
    "    model_weights_save_path = 'C:/Kaggle/BooksPrice/KerasModels/'\n",
    "    file_name = \"20191018_Keras14_Model_Weights_Fold_\"+str(fold_num)+'.h5'\n",
    "    final_path = model_weights_save_path+file_name\n",
    "    print(\"Model Weights File Name : \",final_path)\n",
    "    keras01_Models.append(final_path)\n",
    "    \n",
    "    es = EarlyStopping(mode='min',\n",
    "                       verbose=1,\n",
    "                       patience=10)\n",
    "    \n",
    "    checkpointer = ModelCheckpoint(filepath=final_path,\n",
    "                                   mode='min',\n",
    "                                   verbose=1,\n",
    "                                   save_best_only=True)\n",
    "    \n",
    "    history = model.fit(temp_train_tf, temp_train['Price_Log'].values,\n",
    "                        epochs = 500,\n",
    "                        batch_size = batch_size,\n",
    "                        verbose = 1,\n",
    "                        shuffle = 'batch',\n",
    "                        validation_data = (temp_val_tf, temp_val['Price_Log'].values),\n",
    "                        callbacks = [es,checkpointer])\n",
    "    \n",
    "    \n",
    "    loaded_model = load_model(final_path)\n",
    "    temp_val['Price_Log_Pred_Keras'] = loaded_model.predict(temp_val_tf)\n",
    "    \n",
    "    print('Completed for Fold - ',fold_num)\n",
    "    print('Val Data RMSLE : ',sqrt(mean_squared_error(temp_val['Price_Log'], temp_val['Price_Log_Pred_Keras'])))\n",
    "    \n",
    "    if(IterationNum == 1):\n",
    "        CV_SCORED_DATA = temp_val.copy(deep=True)\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        sub_data_preds = loaded_model.predict(test[FeatureNames].values)\n",
    "    else:\n",
    "        CV_SCORED_DATA = pd.concat([CV_SCORED_DATA,temp_val])\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        sub_data_preds = sub_data_preds + loaded_model.predict(test[FeatureNames].values)\n",
    "\n",
    "    IterationNum = IterationNum + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(keras01_Models)\n",
    "print(\"Keras 01 CV RMSLE = \",sqrt(mean_squared_error(CV_SCORED_DATA['Price_Log'], CV_SCORED_DATA['Price_Log_Pred_Keras'])))\n",
    "print(\"CV 1-RMSLE = \",1-sqrt(mean_squared_error(CV_SCORED_DATA['Price_Log'], CV_SCORED_DATA['Price_Log_Pred_Keras'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_cv_predictions = pd.merge(training_cv_predictions, CV_SCORED_DATA[['id','Price_Log_Pred_Keras']], on='id')\n",
    "training_cv_predictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_data_preds = sub_data_preds / len(fold_list)\n",
    "test['Price_Log_Pred_Keras'] = sub_data_preds\n",
    "test['Price_Log_Pred_Keras'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_cv_predictions.to_csv(\"C:\\\\Kaggle\\\\BooksPrice\\\\CV Scrd Trn Datasets\\\\20191025_Ensemble17_DS.csv\", index=False)\n",
    "test.to_csv(\"C:\\\\Kaggle\\\\BooksPrice\\\\CV Scrd Tst Datasets\\\\20191025_Ensemble17_DS.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import Ridge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "IterationNum = 1\n",
    "for fold_num in fold_list:\n",
    "    print(\"Running CV Iteration Num :\", IterationNum)\n",
    "    \n",
    "    temp_train = train[train['FOLD_NUM'] != fold_num]\n",
    "    temp_val = train[train['FOLD_NUM'] == fold_num]\n",
    "    \n",
    "    temp_train_tf = temp_train[FeatureNames].values\n",
    "    temp_val_tf = temp_val[FeatureNames].values\n",
    "    \n",
    "    ridgereg = Ridge(alpha=0.03,normalize=True)\n",
    "    ridgereg.fit(temp_train_tf,temp_train['Price_Log'])\n",
    "    \n",
    "    temp_val['Price_Log_Pred_RIDGE'] = ridgereg.predict(temp_val_tf)\n",
    "    \n",
    "    print('Completed for Fold - ',fold_num)\n",
    "    print('Val Data RMSLE : ',sqrt(mean_squared_error(temp_val['Price_Log'], temp_val['Price_Log_Pred_RIDGE'])))\n",
    "    \n",
    "    if(IterationNum == 1):\n",
    "        CV_SCORED_DATA = temp_val.copy(deep=True)\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        sub_data_preds = ridgereg.predict(test[FeatureNames].values)\n",
    "    else:\n",
    "        CV_SCORED_DATA = pd.concat([CV_SCORED_DATA,temp_val])\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        sub_data_preds = sub_data_preds + ridgereg.predict(test[FeatureNames].values)\n",
    "\n",
    "    IterationNum = IterationNum + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"RIDGE 01 CV RMSLE = \",sqrt(mean_squared_error(CV_SCORED_DATA['Price_Log'], CV_SCORED_DATA['Price_Log_Pred_RIDGE'])))\n",
    "print(\"CV 1-RMSLE = \",1-sqrt(mean_squared_error(CV_SCORED_DATA['Price_Log'], CV_SCORED_DATA['Price_Log_Pred_RIDGE'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_cv_predictions = pd.merge(training_cv_predictions, CV_SCORED_DATA[['id','Price_Log_Pred_RIDGE']], on='id')\n",
    "training_cv_predictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_data_preds = sub_data_preds / len(fold_list)\n",
    "test['Price_Log_Pred_RIDGE'] = sub_data_preds\n",
    "test['Price_Log_Pred_RIDGE'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import Lasso"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "IterationNum = 1\n",
    "for fold_num in fold_list:\n",
    "    print(\"Running CV Iteration Num :\", IterationNum)\n",
    "    \n",
    "    temp_train = train[train['FOLD_NUM'] != fold_num]\n",
    "    temp_val = train[train['FOLD_NUM'] == fold_num]\n",
    "    \n",
    "    temp_train_tf = temp_train[FeatureNames].values\n",
    "    temp_val_tf = temp_val[FeatureNames].values\n",
    "    \n",
    "    lassoreg = Lasso(alpha=0.000009,normalize=True, max_iter=1e6)\n",
    "    lassoreg.fit(temp_train_tf,temp_train['Price_Log'])\n",
    "    \n",
    "    temp_val['Price_Log_Pred_LASSO'] = lassoreg.predict(temp_val_tf)\n",
    "    \n",
    "    print('Completed for Fold - ',fold_num)\n",
    "    print('Val Data RMSLE : ',sqrt(mean_squared_error(temp_val['Price_Log'], temp_val['Price_Log_Pred_LASSO'])))\n",
    "    \n",
    "    if(IterationNum == 1):\n",
    "        CV_SCORED_DATA = temp_val.copy(deep=True)\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        sub_data_preds = lassoreg.predict(test[FeatureNames].values)\n",
    "    else:\n",
    "        CV_SCORED_DATA = pd.concat([CV_SCORED_DATA,temp_val])\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        sub_data_preds = sub_data_preds + lassoreg.predict(test[FeatureNames].values)\n",
    "\n",
    "    IterationNum = IterationNum + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"LASSO 01 CV RMSLE = \",sqrt(mean_squared_error(CV_SCORED_DATA['Price_Log'], CV_SCORED_DATA['Price_Log_Pred_LASSO'])))\n",
    "print(\"CV 1-RMSLE = \",1-sqrt(mean_squared_error(CV_SCORED_DATA['Price_Log'], CV_SCORED_DATA['Price_Log_Pred_LASSO'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_cv_predictions = pd.merge(training_cv_predictions, CV_SCORED_DATA[['id','Price_Log_Pred_LASSO']], on='id')\n",
    "training_cv_predictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_data_preds = sub_data_preds / len(fold_list)\n",
    "test['Price_Log_Pred_LASSO'] = sub_data_preds\n",
    "test['Price_Log_Pred_LASSO'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "IterationNum = 1\n",
    "for fold_num in fold_list:\n",
    "    print(\"Running CV Iteration Num :\", IterationNum)\n",
    "    \n",
    "    temp_train = train[train['FOLD_NUM'] != fold_num]\n",
    "    temp_val = train[train['FOLD_NUM'] == fold_num]\n",
    "    \n",
    "    temp_train_tf = temp_train[FeatureNames].values\n",
    "    temp_val_tf = temp_val[FeatureNames].values\n",
    "    \n",
    "    KNN = KNeighborsRegressor(n_neighbors = 51)\n",
    "    KNN.fit(temp_train_tf,temp_train['Price_Log'])\n",
    "    \n",
    "    temp_val['Price_Log_Pred_KNN'] = KNN.predict(temp_val_tf)\n",
    "    \n",
    "    print('Completed for Fold - ',fold_num)\n",
    "    print('Val Data RMSLE : ',sqrt(mean_squared_error(temp_val['Price_Log'], temp_val['Price_Log_Pred_KNN'])))\n",
    "    \n",
    "    if(IterationNum == 1):\n",
    "        CV_SCORED_DATA = temp_val.copy(deep=True)\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        sub_data_preds = KNN.predict(test[FeatureNames].values)\n",
    "    else:\n",
    "        CV_SCORED_DATA = pd.concat([CV_SCORED_DATA,temp_val])\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        sub_data_preds = sub_data_preds + KNN.predict(test[FeatureNames].values)\n",
    "\n",
    "    IterationNum = IterationNum + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"KNN 01 CV RMSLE = \",sqrt(mean_squared_error(CV_SCORED_DATA['Price_Log'], CV_SCORED_DATA['Price_Log_Pred_KNN'])))\n",
    "print(\"CV 1-RMSLE = \",1-sqrt(mean_squared_error(CV_SCORED_DATA['Price_Log'], CV_SCORED_DATA['Price_Log_Pred_KNN'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_cv_predictions = pd.merge(training_cv_predictions, CV_SCORED_DATA[['id','Price_Log_Pred_KNN']], on='id')\n",
    "training_cv_predictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_data_preds = sub_data_preds / len(fold_list)\n",
    "test['Price_Log_Pred_KNN'] = sub_data_preds\n",
    "test['Price_Log_Pred_KNN'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_cv_predictions.to_csv(\"C:\\\\Kaggle\\\\BooksPrice\\\\CV Scrd Trn Datasets\\\\20191025_Ensemble17_DS.csv\", index=False)\n",
    "test.to_csv(\"C:\\\\Kaggle\\\\BooksPrice\\\\CV Scrd Tst Datasets\\\\20191025_Ensemble17_DS.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "IterationNum = 1\n",
    "for fold_num in fold_list:\n",
    "    print(\"Running CV Iteration Num :\", IterationNum)\n",
    "    \n",
    "    temp_train = train[train['FOLD_NUM'] != fold_num]\n",
    "    temp_val = train[train['FOLD_NUM'] == fold_num]\n",
    "    \n",
    "    temp_train_tf = temp_train[FeatureNames].values\n",
    "    temp_val_tf = temp_val[FeatureNames].values\n",
    "    \n",
    "    RF = RandomForestRegressor(n_estimators = 100, min_samples_leaf = 1, max_features = 9, random_state = 123,\n",
    "                               verbose = 1, max_depth = 40)\n",
    "    RF.fit(temp_train_tf,temp_train['Price_Log'])\n",
    "    \n",
    "    temp_val['Price_Log_Pred_RF'] = RF.predict(temp_val_tf)\n",
    "    \n",
    "    print('Completed for Fold - ',fold_num)\n",
    "    print('Val Data RMSLE : ',sqrt(mean_squared_error(temp_val['Price_Log'], temp_val['Price_Log_Pred_RF'])))\n",
    "    \n",
    "    if(IterationNum == 1):\n",
    "        CV_SCORED_DATA = temp_val.copy(deep=True)\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        sub_data_preds = RF.predict(test[FeatureNames].values)\n",
    "    else:\n",
    "        CV_SCORED_DATA = pd.concat([CV_SCORED_DATA,temp_val])\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        sub_data_preds = sub_data_preds + RF.predict(test[FeatureNames].values)\n",
    "\n",
    "    IterationNum = IterationNum + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"RF 01 CV RMSLE = \",sqrt(mean_squared_error(CV_SCORED_DATA['Price_Log'], CV_SCORED_DATA['Price_Log_Pred_RF'])))\n",
    "print(\"CV 1-RMSLE = \",1-sqrt(mean_squared_error(CV_SCORED_DATA['Price_Log'], CV_SCORED_DATA['Price_Log_Pred_RF'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_cv_predictions = pd.merge(training_cv_predictions, CV_SCORED_DATA[['id','Price_Log_Pred_RF']], on='id')\n",
    "training_cv_predictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_data_preds = sub_data_preds / len(fold_list)\n",
    "test['Price_Log_Pred_RF'] = sub_data_preds\n",
    "test['Price_Log_Pred_RF'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_cv_predictions.to_csv(\"C:\\\\Kaggle\\\\BooksPrice\\\\CV Scrd Trn Datasets\\\\20191025_Ensemble17_DS.csv\", index=False)\n",
    "test.to_csv(\"C:\\\\Kaggle\\\\BooksPrice\\\\CV Scrd Tst Datasets\\\\20191025_Ensemble17_DS.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "IterationNum = 1\n",
    "for fold_num in fold_list:\n",
    "    print(\"Running CV Iteration Num :\", IterationNum)\n",
    "    \n",
    "    temp_train = train[train['FOLD_NUM'] != fold_num]\n",
    "    temp_val = train[train['FOLD_NUM'] == fold_num]\n",
    "    \n",
    "    temp_train_tf = temp_train[FeatureNames].values\n",
    "    temp_val_tf = temp_val[FeatureNames].values\n",
    "    \n",
    "    dtrain = xgb.DMatrix(data = temp_train_tf, label = temp_train['Price_Log'])\n",
    "    dtest = xgb.DMatrix(data = temp_val_tf, label = temp_val['Price_Log'])\n",
    "                \n",
    "    watchlist = [(dtrain, 'train'), (dtest, 'eval')]\n",
    "    \n",
    "    params1 = { 'seed': 111,\n",
    "                'colsample_bytree': 0.8,\n",
    "                'verbosity': 1,\n",
    "                'subsample': 0.3,\n",
    "                'learning_rate': 0.01,\n",
    "                'objective': 'reg:squarederror',\n",
    "                'max_depth': 3,\n",
    "                'min_child_weight': 1,\n",
    "                'booster': 'gbtree',\n",
    "                'eval_metric': 'rmse' }\n",
    "    \n",
    "    XGB = xgb.train(params = params1,\n",
    "                    dtrain = dtrain,\n",
    "                    num_boost_round = 1000000,\n",
    "                    evals = watchlist,\n",
    "                    verbose_eval = 50,\n",
    "                    early_stopping_rounds = 300)\n",
    "    \n",
    "    temp_val['Price_Log_Pred_XGB'] = XGB.predict(xgb.DMatrix(data = temp_val_tf), ntree_limit = XGB.best_ntree_limit)\n",
    "    \n",
    "    print('Completed for Fold - ',fold_num)\n",
    "    print('Val Data RMSLE : ',sqrt(mean_squared_error(temp_val['Price_Log'], temp_val['Price_Log_Pred_XGB'])))\n",
    "    \n",
    "    if(IterationNum == 1):\n",
    "        CV_SCORED_DATA = temp_val.copy(deep=True)\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        sub_data_preds = XGB.predict(xgb.DMatrix(data = test[FeatureNames].values), ntree_limit = XGB.best_ntree_limit)\n",
    "    else:\n",
    "        CV_SCORED_DATA = pd.concat([CV_SCORED_DATA,temp_val])\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        sub_data_preds = sub_data_preds + XGB.predict(xgb.DMatrix(data = test[FeatureNames].values), ntree_limit = XGB.best_ntree_limit)\n",
    "\n",
    "    IterationNum = IterationNum + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"XGB 01 CV RMSLE = \",sqrt(mean_squared_error(CV_SCORED_DATA['Price_Log'], CV_SCORED_DATA['Price_Log_Pred_XGB'])))\n",
    "print(\"CV 1-RMSLE = \",1-sqrt(mean_squared_error(CV_SCORED_DATA['Price_Log'], CV_SCORED_DATA['Price_Log_Pred_XGB'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_cv_predictions = pd.merge(training_cv_predictions, CV_SCORED_DATA[['id','Price_Log_Pred_XGB']], on='id')\n",
    "training_cv_predictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_data_preds = sub_data_preds / len(fold_list)\n",
    "test['Price_Log_Pred_XGB'] = sub_data_preds\n",
    "test['Price_Log_Pred_XGB'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_cv_predictions.to_csv(\"C:\\\\Kaggle\\\\BooksPrice\\\\CV Scrd Trn Datasets\\\\20191025_Ensemble17_DS.csv\", index=False)\n",
    "test.to_csv(\"C:\\\\Kaggle\\\\BooksPrice\\\\CV Scrd Tst Datasets\\\\20191025_Ensemble17_DS.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_names_ensemble = list(training_cv_predictions.columns[4:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corr = training_cv_predictions[feature_names_ensemble + ['Price_Log']].corr()\n",
    "ax = sns.heatmap(\n",
    "    corr, \n",
    "    vmin=0, vmax=1, center=0.5,\n",
    "    cmap=sns.diverging_palette(20, 220, n=200),\n",
    "    square=True\n",
    ")\n",
    "ax.set_xticklabels(\n",
    "    ax.get_xticklabels(),\n",
    "    rotation=45,\n",
    "    horizontalalignment='right'\n",
    ");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "IterationNum = 1\n",
    "for fold_num in fold_list:\n",
    "    print(\"Running CV Iteration Num :\", IterationNum)\n",
    "    \n",
    "    temp_train = training_cv_predictions[training_cv_predictions['FOLD_NUM'] != fold_num]\n",
    "    temp_val = training_cv_predictions[training_cv_predictions['FOLD_NUM'] == fold_num]\n",
    "    \n",
    "    ridgereg_e = Ridge(alpha=0.002,normalize=True)\n",
    "    ridgereg_e.fit(temp_train[feature_names_ensemble],temp_train['Price_Log'])\n",
    "    \n",
    "    temp_val['Price_Log_Pred_LGB_ENS'] = ridgereg_e.predict(temp_val[feature_names_ensemble])\n",
    "    \n",
    "    print(\"Fold RMSLE = \",sqrt(mean_squared_error(temp_val['Price_Log'], temp_val['Price_Log_Pred_LGB_ENS'])))\n",
    "    \n",
    "    if(IterationNum == 1):\n",
    "        CV_SCORED_DATA = temp_val.copy(deep=True)\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        sub_data_preds = ridgereg_e.predict(test[feature_names_ensemble])\n",
    "    else:\n",
    "        CV_SCORED_DATA = pd.concat([CV_SCORED_DATA,temp_val])\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        sub_data_preds = sub_data_preds + ridgereg_e.predict(test[feature_names_ensemble])\n",
    "\n",
    "    IterationNum = IterationNum + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"LGB ENS CV RMSLE = \",sqrt(mean_squared_error(CV_SCORED_DATA['Price_Log'], CV_SCORED_DATA['Price_Log_Pred_LGB_ENS'])))\n",
    "print(\"CV 1-RMSLE = \",1-sqrt(mean_squared_error(CV_SCORED_DATA['Price_Log'], CV_SCORED_DATA['Price_Log_Pred_LGB_ENS'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_cv_predictions = pd.merge(training_cv_predictions, CV_SCORED_DATA[['id','Price_Log_Pred_LGB_ENS']], on='id')\n",
    "training_cv_predictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_data_preds = sub_data_preds / len(fold_list)\n",
    "test['Price_Log_Pred_LGB_ENS'] = sub_data_preds\n",
    "test['Price_Log_Pred_LGB_ENS'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "IterationNum = 1\n",
    "for fold_num in fold_list:\n",
    "    print(\"Running CV Iteration Num :\", IterationNum)\n",
    "    \n",
    "    temp_train = training_cv_predictions[training_cv_predictions['FOLD_NUM'] != fold_num]\n",
    "    temp_val = training_cv_predictions[training_cv_predictions['FOLD_NUM'] == fold_num]\n",
    "    \n",
    "    trn_data = lgb.Dataset(temp_train[feature_names_ensemble], label=temp_train['Price_Log'])\n",
    "    val_data = lgb.Dataset(temp_val[feature_names_ensemble], label=temp_val['Price_Log'])\n",
    "    \n",
    "    param = {\n",
    "        'learning_rate': 0.001,\n",
    "        'max_depth': 3,\n",
    "        'min_data_in_leaf': 1,\n",
    "        'bagging_freq': 5,\n",
    "        'bagging_fraction': 0.3,\n",
    "        'feature_fraction': 0.3,\n",
    "        'boost': 'gbdt',    \n",
    "        'objective': 'regression',\n",
    "        'metric': 'rmse',\n",
    "        'seed': 392\n",
    "    }\n",
    "    \n",
    "    model = lgb.train(param, trn_data, 2000000,valid_sets = val_data, verbose_eval=50, early_stopping_rounds = 300)\n",
    "    \n",
    "    temp_val['Price_Log_Pred_RIDGE_ENS'] = model.predict(temp_val[feature_names_ensemble], num_iteration=model.best_iteration)\n",
    "    \n",
    "    print(\"Fold RMSLE = \",sqrt(mean_squared_error(temp_val['Price_Log'], temp_val['Price_Log_Pred_RIDGE_ENS'])))\n",
    "    \n",
    "    if(IterationNum == 1):\n",
    "        CV_SCORED_DATA = temp_val.copy(deep=True)\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        sub_data_preds = model.predict(test[feature_names_ensemble].values, num_iteration=model.best_iteration)\n",
    "    else:\n",
    "        CV_SCORED_DATA = pd.concat([CV_SCORED_DATA,temp_val])\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        sub_data_preds = sub_data_preds + model.predict(test[feature_names_ensemble].values, num_iteration=model.best_iteration)\n",
    "\n",
    "    IterationNum = IterationNum + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"LGB ENS CV RMSLE = \",sqrt(mean_squared_error(CV_SCORED_DATA['Price_Log'], CV_SCORED_DATA['Price_Log_Pred_RIDGE_ENS'])))\n",
    "print(\"CV 1-RMSLE = \",1-sqrt(mean_squared_error(CV_SCORED_DATA['Price_Log'], CV_SCORED_DATA['Price_Log_Pred_RIDGE_ENS'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_cv_predictions = pd.merge(training_cv_predictions, CV_SCORED_DATA[['id','Price_Log_Pred_RIDGE_ENS']], on='id')\n",
    "training_cv_predictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_data_preds = sub_data_preds / len(fold_list)\n",
    "test['Price_Log_Pred_RIDGE_ENS'] = sub_data_preds\n",
    "test['Price_Log_Pred_RIDGE_ENS'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_cv_predictions.to_csv(\"C:\\\\Kaggle\\\\BooksPrice\\\\CV Scrd Trn Datasets\\\\20191025_Ensemble17_DS.csv\", index=False)\n",
    "test.to_csv(\"C:\\\\Kaggle\\\\BooksPrice\\\\CV Scrd Tst Datasets\\\\20191025_Ensemble17_DS.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try_ens = 0.25*training_cv_predictions['Price_Log_Pred_RIDGE_ENS'] + 0.75*training_cv_predictions['Price_Log_Pred_LGB_ENS']\n",
    "\n",
    "print(\"ENS RMSLE = \",sqrt(mean_squared_error(training_cv_predictions['Price_Log'], try_ens)))\n",
    "print(\"1-RMSLE = \",1-sqrt(mean_squared_error(training_cv_predictions['Price_Log'], try_ens)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_cv_predictions['Price_Log_Pred_Final_Ens'] = try_ens\n",
    "test['Price_Log_Pred_Final_Ens'] = 0.25*test['Price_Log_Pred_RIDGE_ENS'] + 0.75*test['Price_Log_Pred_LGB_ENS']\n",
    "\n",
    "training_cv_predictions.to_csv(\"C:\\\\Kaggle\\\\BooksPrice\\\\CV Scrd Trn Datasets\\\\20191025_Ensemble17_DS.csv\", index=False)\n",
    "test.to_csv(\"C:\\\\Kaggle\\\\BooksPrice\\\\CV Scrd Tst Datasets\\\\20191025_Ensemble17_DS.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.read_excel('C:/Kaggle/BooksPrice/Participants_Data/Sample_Submission.xlsx', encoding='ISO-8859-1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_data_preds2 = (10**test['Price_Log_Pred_Final_Ens'].values) - 1\n",
    "pd.DataFrame(sub_data_preds2).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission['Price'] = sub_data_preds2\n",
    "submission.to_excel('C:\\\\Kaggle\\\\BooksPrice\\\\Submissions\\\\20191025_Ensemble17_DS.xlsx', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
